# -*- coding: utf-8 -*-
import scrapy
from fang.items import FangItem
import sys
reload(sys)   
sys.setdefaultencoding('utf8') 


class ShanghaiHouseSpider(scrapy.Spider):
	name = 'shanghai_house'
	allowed_domains = ['fang.com']
	start_urls = ['http://esf.sh.fang.com/housing/18__0_0_0_0_1_0_0/']

	def start_requests(self):
		#url = 'http://esf.sh.fang.com/housing/18__0_0_0_0_1_0_0/'
		for i in range(1,79):#修改成79，上海地区小区共有78页
			url = 'http://esf.sh.fang.com/housing/18__0_0_0_0_' + str(i) + '_0_0/'
			yield scrapy.Request(url,callback=self.parse_item)

	def parse_item(self,response):
		link = response.xpath('//div[@class="houseList"]/div/dl/dd/p[1]/a/@href').extract()
		print len(link)
		for url in link:
			print "正在爬取链接："
			print url
			#print "****************parse_item"
			yield scrapy.Request(url,callback=self.parse)

	def parse(self, response):
		print "正在提取数据%s"%response.url
		item = FangItem()
		#print "********parse"
		#小区名称
		#ceshi = response.xpath('//*[@id="xqw_B01_12"]/text()').extract()
		#print ceshi

		#名称
		#item['name'] = response.xpath('//*[@id="xqw_B01_12"]/text()').extract()
		item['name'] = response.xpath('/html/head/title/text()').extract()
		#所在区域
		item['address'] = response.xpath('/html/body/div[3]/div[4]/div[1]/div[2]/div[2]/ul[1]/li[1]/text()').extract()
		#楼栋总数
		item['sum_loudong'] = response.xpath('/html/body/div[3]/div[4]/div[1]/div[2]/div[2]/ul[1]/li[2]/text()').extract()

		#建筑年代
		item['build_year'] = response.xpath('/html/body/div[3]/div[4]/div[1]/div[2]/div[2]/ul[1]/li[3]/text()').extract()

		#总户数
		item['sum_hushu'] = response.xpath('/html/body/div[3]/div[4]/div[1]/div[2]/div[2]/ul[1]/li[4]/text()').extract()

		#物业类型
		item['type_wuye'] = response.xpath('/html/body/div[3]/div[4]/div[1]/div[2]/div[2]/ul[1]/li[5]/text()').extract()

		#容积率
		item['rongjilv'] = response.xpath('/html/body/div[3]/div[4]/div[1]/div[2]/div[2]/ul[1]/li[6]/text()').extract()

		#绿化率
		item['green'] = response.xpath('/html/body/div[3]/div[4]/div[1]/div[2]/div[2]/ul[1]/li[7]/text()').extract()

		#物业费
		item['wuye_price'] = response.xpath('/html/body/div[3]/div[4]/div[1]/div[2]/div[2]/ul[1]/li[8]/text()').extract()

		#开发商
		item['build_name'] = response.xpath('/html/body/div[3]/div[4]/div[1]/div[2]/div[2]/ul[1]/li[9]/text()').extract()

		#物业公司
		item['wuye_name'] = response.xpath('/html/body/div[3]/div[4]/div[1]/div[2]/div[2]/ul[1]/li[10]/text()').extract()
		#if item['wuye_name']:
		#	item['wuye_name'] == ["None"]
		#小区地址
		item['place_xiaoqu'] = response.xpath('//*[@id="xqw_C03_03"]/text()').extract()

		#周边学校
		item['school'] = response.xpath('/html/body/div[3]/div[4]/div[1]/div[2]/div[2]/ul[1]/li[12]/div/span/a/text()').extract()
		#if item['school']:
		#	item['school'] == ["None"]
		#小区均价
		item['price'] = response.xpath('/html/body/div[3]/div[4]/div[1]/div[2]/div[2]/ul[2]/li[1]/em/text()').extract()

		#环比上月（上涨or下跌）
		item['com_last_month'] = response.xpath('/html/body/div[3]/div[4]/div[1]/div[2]/div[2]/ul[2]/li[1]/span/text()').extract()

		#小区成交
		item['turnover'] = response.xpath('/html/body/div[3]/div[4]/div[1]/div[2]/div[2]/ul[2]/li[2]/em/text()').extract()


		return item


